---
title: "Chatbot Paper"
date: "2025-08-06"
output: html_document
---

```{r pressure, echo=FALSE}

# LOAD PACKAGES

library(dplyr)
library(brglm2)
library(stargazer)
library(tidyverse)
library(reshape2)
library(pheatmap)
library(viridis)
library(reshape2)
library(ggpubr)
library(patchwork)

```

```{r setup, include=FALSE}

# DATA PREPARATION

knitr::opts_chunk$set(echo = FALSE)

merged <- read.csv("merged.csv", sep = ";")
comparison <- read.csv("comparison.csv", sep = ";")

df <- data_frame(merged)
df_2 <- data_frame(comparison)

df$fake[df$fake == 'No'] <- 'no'
df$fake[df$fake == 'Yes'] <- 'yes'
df$propaganda[df$propaganda == 'No'] <- 'no'
df$propaganda[df$propaganda == ' no'] <- 'no'
df$propaganda[df$propaganda == 'Yes'] <- 'yes'
df$pravda[df$pravda == 'No'] <- 'no'
df$pravda[df$pravda == 'Yes'] <- 'yes'
df$model[df$model == 'Gemini'] <- 'Gemini 2.5 Flash'


df$fake <- as.factor(df$fake)
df$propaganda <- as.factor(df$propaganda)
df$pravda <- as.factor(df$pravda)
df$prompt_type <- as.factor(df$prompt_type)
df$propaganda <- droplevels(df$propaganda)



```


```{r pressure, echo=FALSE}

# Supporting disinfo

plot_df <- df %>%
  mutate(
    fake    = tolower(trimws(fake)),
    model   = factor(model, levels = c("ChatGPT-4o", "Copilot", "Gemini 2.5 Flash", "Grok-2")),
    country = recode(country,
                     "UK" = "United Kingdom",
                     "CH" = "Switzerland",
                     .default = country),
    country = factor(country, levels = c("United Kingdom", "Switzerland")),
    fake = recode(fake,
                  "yes" = "Supporting disinformation",
                  "no"  = "Not supporting disinformation",
                  .default = NA_character_),
    fake = factor(fake, levels = c("Supporting disinformation", "Not supporting disinformation"))
  ) %>%
  filter(!is.na(fake), !is.na(model), !is.na(country)) %>%  
  count(country, model, fake, name = "n") %>%
  group_by(country, model) %>%
  mutate(percentage = 100 * n / sum(n)) %>%
  ungroup()

graph_fake <- ggplot(plot_df, aes(x = model, y = percentage, fill = fake)) +
  geom_col(width = 0.8, color = "white", linewidth = 0.2) +
  facet_grid(. ~ country, scales = "free_x", space = "free_x") +
  scale_fill_manual(
    values = c("Supporting disinformation" = "red",
               "Not supporting disinformation" = "blue"),
    name = "Output"
  ) +
  labs(
    x = NULL, y = "Percentage of outputs",
    title = NULL
  ) +
  theme_minimal(base_size = 12) +
  theme(
    panel.grid.major.x = element_blank(),
    axis.text.x = element_text(angle = 25, hjust = .5),
    strip.text = element_text(face = "bold", size = 16),
    legend.position = "bottom",
    legend.title = element_text(face = "bold")
  )

graph_fake 

```


```{r pressure, echo=FALSE}

# Pravda links

plot_df_2 <- df %>%
  mutate(
    pravda    = tolower(trimws(pravda)),
    model   = factor(model, levels = c("ChatGPT-4o", "Copilot", "Gemini 2.5 Flash", "Grok-2")),
    country = recode(country,
                     "UK" = "United Kingdom",
                     "CH" = "Switzerland",
                     .default = country),
    country = factor(country, levels = c("United Kingdom", "Switzerland")),
    pravda = recode(pravda,
                  "yes" = "With Pravda links",
                  "no"  = "Without Pravda links",
                  .default = NA_character_),
    pravda = factor(pravda, levels = c("With Pravda links", "Without Pravda links"))
  ) %>%
  filter(!is.na(pravda), !is.na(model), !is.na(country)) %>%  
  count(country, model, pravda, name = "n") %>%
  group_by(country, model) %>%
  mutate(percentage = 100 * n / sum(n)) %>%
  ungroup()

graph_pravda <- ggplot(plot_df_2, aes(x = model, y = percentage, fill = pravda)) +
  geom_col(width = 0.8, color = "white", linewidth = 0.2) +
  facet_grid(. ~ country, scales = "free_x", space = "free_x") +
  scale_fill_manual(
    values = c("With Pravda links" = "red",
               "Without Pravda links" = "blue"),
    name = "Output"
  ) +
  labs(
    x = NULL, y = "Percentage of outputs",
    title = NULL
  ) +
  theme_minimal(base_size = 12) +
  theme(
    panel.grid.major.x = element_blank(),
    axis.text.x = element_text(angle = 25, hjust = .5),
    strip.text = element_text(face = "bold", size = 16),
    legend.position = "bottom",
    legend.title = element_text(face = "bold")
  )

```



```{r pressure, echo=FALSE}

# Prompt type


plot_df3 <- df %>%
  mutate(
    model = factor(model, levels = c("ChatGPT-4o", "Copilot", "Gemini 2.5 Flash", "Grok-2")),
    pravda = tolower(trimws(pravda)),
    pravda = recode(pravda,
                    "yes" = "With Pravda links",
                    "no"  = "Without Pravda links",
                    .default = NA_character_),
    pravda = factor(pravda, levels = c("With Pravda links", "Without Pravda links")),
    prompt_type = recode(prompt_type,
                         "general"  = "General prompts",
                         "specific" = "Specific prompts")
  ) %>%
  filter(!is.na(pravda), !is.na(model), !is.na(prompt_type)) %>%
  count(prompt_type, model, pravda, name = "n") %>%
  group_by(prompt_type, model) %>%
  mutate(percentage = 100 * n / sum(n)) %>%
  ungroup()

pravda_graph <- ggplot(plot_df3, aes(x = model, y = percentage, fill = pravda)) +
    geom_bar(stat = "identity", position = position_stack(), width = 0.9) +
    facet_wrap(~prompt_type) +
    labs(
        y = "Percentage of outputs",
        x = "Chatbot model",
        fill = "Output"
    ) +
    scale_fill_manual(values = c(
        "With Pravda links"    = "red",
        "Without Pravda links" = "blue"
    )) +
    theme_minimal(base_size = 12) +
    theme(
        panel.grid.major.x = element_blank(),
        axis.text.x  = element_text(angle = 25, hjust = 0.5),
        strip.text   = element_text(face = "bold", size = 16),   
        legend.position = "bottom",
        legend.title   = element_text(face = "bold"),
        axis.title.x   = element_blank()
    )


pravda_graph

```


```{r pressure, echo=FALSE}

# Russian propaganda links

plot_df_4 <- df %>%
  mutate(
    propaganda    = tolower(trimws(propaganda)),
    model   = factor(model, levels = c("ChatGPT-4o", "Copilot", "Gemini 2.5 Flash", "Grok-2")),
    country = recode(country,
                     "UK" = "United Kingdom",
                     "CH" = "Switzerland",
                     .default = country),
    country = factor(country, levels = c("United Kingdom", "Switzerland")),
    propaganda = recode(propaganda,
                  "yes" = "Links to Kremlin-controlled sources",
                  "no"  = "Without links to Kremlin-controlled sources",
                  .default = NA_character_),
    propaganda = factor(propaganda, levels = c("Links to Kremlin-controlled sources", "Without links to Kremlin-controlled sources"))
  ) %>%
  filter(!is.na(propaganda), !is.na(model), !is.na(country)) %>%  
  count(country, model, propaganda, name = "n") %>%
  group_by(country, model) %>%
  mutate(percentage = 100 * n / sum(n)) %>%
  ungroup()

graph_propaganda <- ggplot(plot_df_4, aes(x = model, y = percentage, fill = propaganda)) +
  geom_col(width = 0.8, color = "white", linewidth = 0.2) +
  facet_grid(. ~ country, scales = "free_x", space = "free_x") +
  scale_fill_manual(
    values = c("Links to Kremlin-controlled sources" = "red",
               "Without links to Kremlin-controlled sources" = "blue"),
    name = "Output"
  ) +
  labs(
    x = NULL, y = "Percentage of outputs",
    title = NULL
  ) +
  theme_minimal(base_size = 12) +
  theme(
    panel.grid.major.x = element_blank(),
    axis.text.x = element_text(angle = 25, hjust = 0.5),
    strip.text = element_text(face = "bold", size = 16),
    legend.position = "bottom",
    legend.title = element_text(face = "bold")
  )

```


```{r pressure, echo=FALSE}

# Hamming loss supporting disinfo -- heatmaps

# Wide format

df2 <- df

df2 <- df2 %>%
  mutate(
    fake_num       = ifelse(fake == "yes", 1, 0),
    pravda_num     = ifelse(pravda == "yes", 1, 0),
    propaganda_num = ifelse(propaganda == "yes", 1, 0),
    model_clean = model %>%
      gsub("^ChatGPT-4o$", "GPT", .) %>%
      gsub("^Grok-2$", "Grok", .) %>%
      gsub("^Gemini 2.5 Flash$", "Gemini", .),
    agent = paste(model_clean,
                  country,
                  paste0("Agent-", instance),
                  sep = "-")
  )


df_wide_fake <- df2 %>%
  select(agent, fake_num, prompt) %>%
  pivot_wider(names_from = agent, values_from = fake_num)

df_wide_pravda <- df2 %>%
  select(agent, pravda_num, prompt) %>%
  pivot_wider(names_from = agent, values_from = pravda_num)

df_wide_propaganda <- df2 %>%
  select(agent, propaganda_num, prompt) %>%
  pivot_wider(names_from = agent, values_from = propaganda_num)

# GPT -- supporting disinfo

agents_gpt_fake <- grep("GPT", names(df_wide_fake), value = TRUE)

mat_gpt_fake <- matrix(0, nrow = length(agents_gpt_fake), ncol = length(agents_gpt_fake),
                       dimnames = list(agents_gpt_fake, agents_gpt_fake))

for (i in 1:length(agents_gpt_fake)) {
  for (j in 1:length(agents_gpt_fake)) {
    x <- df_wide_fake[[agents_gpt_fake[i]]]
    y <- df_wide_fake[[agents_gpt_fake[j]]]
    mat_gpt_fake[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_gpt_fake <- as.data.frame(as.table(mat_gpt_fake))

loss_gpt_fake <- ggplot(hamming_gpt_fake, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "GPT agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Gemini -- supporting disinfo

agents_gemini_fake <- grep("Gemini", names(df_wide_fake), value = TRUE)

mat_gemini_fake <- matrix(0, nrow = length(agents_gemini_fake), ncol = length(agents_gemini_fake),
                       dimnames = list(agents_gemini_fake, agents_gemini_fake))

for (i in 1:length(agents_gemini_fake)) {
  for (j in 1:length(agents_gemini_fake)) {
    x <- df_wide_fake[[agents_gemini_fake[i]]]
    y <- df_wide_fake[[agents_gemini_fake[j]]]
    mat_gemini_fake[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_gemini_fake <- as.data.frame(as.table(mat_gemini_fake))

loss_gemini_fake <- ggplot(hamming_gemini_fake, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Gemini agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Copilot -- supporting disinfo

agents_copilot_fake <- grep("Copilot", names(df_wide_fake), value = TRUE)

mat_copilot_fake <- matrix(0, nrow = length(agents_copilot_fake), ncol = length(agents_copilot_fake),
                       dimnames = list(agents_copilot_fake, agents_copilot_fake))

for (i in 1:length(agents_copilot_fake)) {
  for (j in 1:length(agents_copilot_fake)) {
    x <- df_wide_fake[[agents_copilot_fake[i]]]
    y <- df_wide_fake[[agents_copilot_fake[j]]]
    mat_copilot_fake[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_copilot_fake <- as.data.frame(as.table(mat_copilot_fake))

loss_copilot_fake <- ggplot(hamming_copilot_fake, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Copilot agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))


# Grok -- supporting disinfo

agents_grok_fake <- grep("Grok", names(df_wide_fake), value = TRUE)

mat_grok_fake <- matrix(0, nrow = length(agents_grok_fake), ncol = length(agents_grok_fake),
                       dimnames = list(agents_grok_fake, agents_grok_fake))

for (i in 1:length(agents_grok_fake)) {
  for (j in 1:length(agents_grok_fake)) {
    x <- df_wide_fake[[agents_grok_fake[i]]]
    y <- df_wide_fake[[agents_grok_fake[j]]]
    mat_grok_fake[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_grok_fake <- as.data.frame(as.table(mat_grok_fake))

loss_grok_fake <- ggplot(hamming_grok_fake, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Grok agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Merge -- supporting disinfo

hamming_fake <- ggarrange(loss_gpt_fake, loss_gemini_fake, 
                          loss_copilot_fake, loss_grok_fake,
                          ncol = 2,
                          nrow = 2)

hamming_fake

```

```{r pressure, echo=FALSE}

# Hamming loss Pravda links -- heatmaps

# GPT -- Pravda links

agents_gpt_pravda <- grep("GPT", names(df_wide_pravda), value = TRUE)

mat_gpt_pravda <- matrix(0, nrow = length(agents_gpt_pravda), ncol = length(agents_gpt_pravda),
                       dimnames = list(agents_gpt_pravda, agents_gpt_pravda))

for (i in 1:length(agents_gpt_pravda)) {
  for (j in 1:length(agents_gpt_pravda)) {
    x <- df_wide_pravda[[agents_gpt_pravda[i]]]
    y <- df_wide_pravda[[agents_gpt_pravda[j]]]
    mat_gpt_pravda[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_gpt_pravda <- as.data.frame(as.table(mat_gpt_pravda))

loss_gpt_pravda <- ggplot(hamming_gpt_pravda, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "GPT agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Gemini -- Pravda links

agents_gemini_pravda <- grep("Gemini", names(df_wide_pravda), value = TRUE)

mat_gemini_pravda <- matrix(0, nrow = length(agents_gemini_pravda), ncol = length(agents_gemini_pravda),
                       dimnames = list(agents_gemini_pravda, agents_gemini_pravda))

for (i in 1:length(agents_gemini_pravda)) {
  for (j in 1:length(agents_gemini_pravda)) {
    x <- df_wide_pravda[[agents_gemini_pravda[i]]]
    y <- df_wide_pravda[[agents_gemini_pravda[j]]]
    mat_gemini_pravda[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_gemini_pravda <- as.data.frame(as.table(mat_gemini_pravda))

loss_gemini_pravda <- ggplot(hamming_gemini_pravda, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Gemini agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Copilot -- Pravda links

agents_copilot_pravda <- grep("Copilot", names(df_wide_pravda), value = TRUE)

mat_copilot_pravda <- matrix(0, nrow = length(agents_copilot_pravda), ncol = length(agents_copilot_pravda),
                       dimnames = list(agents_copilot_pravda, agents_copilot_pravda))

for (i in 1:length(agents_copilot_pravda)) {
  for (j in 1:length(agents_copilot_pravda)) {
    x <- df_wide_pravda[[agents_copilot_pravda[i]]]
    y <- df_wide_pravda[[agents_copilot_pravda[j]]]
    mat_copilot_pravda[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_copilot_pravda <- as.data.frame(as.table(mat_copilot_pravda))

loss_copilot_pravda <- ggplot(hamming_copilot_pravda, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Copilot agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))


# Grok -- Pravda links

agents_grok_pravda <- grep("Grok", names(df_wide_pravda), value = TRUE)

mat_grok_pravda <- matrix(0, nrow = length(agents_grok_pravda), ncol = length(agents_grok_pravda),
                       dimnames = list(agents_grok_pravda, agents_grok_pravda))

for (i in 1:length(agents_grok_pravda)) {
  for (j in 1:length(agents_grok_pravda)) {
    x <- df_wide_pravda[[agents_grok_pravda[i]]]
    y <- df_wide_pravda[[agents_grok_pravda[j]]]
    mat_grok_pravda[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_grok_pravda <- as.data.frame(as.table(mat_grok_pravda))

loss_grok_pravda <- ggplot(hamming_grok_pravda, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Grok agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Merge -- Pravda links

hamming_pravda <- ggarrange(loss_gpt_pravda, loss_gemini_pravda, 
                          loss_copilot_pravda, loss_grok_pravda,
                          ncol = 2,
                          nrow = 2)

hamming_pravda

```

```{r pressure, echo=FALSE}

# Hamming loss propaganda links -- heatmaps

# GPT -- propaganda links

agents_gpt_propaganda <- grep("GPT", names(df_wide_propaganda), value = TRUE)

mat_gpt_propaganda <- matrix(0, nrow = length(agents_gpt_propaganda), ncol = length(agents_gpt_propaganda),
                       dimnames = list(agents_gpt_propaganda, agents_gpt_propaganda))

for (i in 1:length(agents_gpt_propaganda)) {
  for (j in 1:length(agents_gpt_propaganda)) {
    x <- df_wide_propaganda[[agents_gpt_propaganda[i]]]
    y <- df_wide_propaganda[[agents_gpt_propaganda[j]]]
    mat_gpt_propaganda[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_gpt_propaganda <- as.data.frame(as.table(mat_gpt_propaganda))

loss_gpt_propaganda <- ggplot(hamming_gpt_propaganda, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "GPT agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Gemini -- propaganda links

agents_gemini_propaganda <- grep("Gemini", names(df_wide_propaganda), value = TRUE)

mat_gemini_propaganda <- matrix(0, nrow = length(agents_gemini_propaganda), 
                                ncol = length(agents_gemini_propaganda),
                                dimnames = list(agents_gemini_propaganda, agents_gemini_propaganda))

for (i in 1:length(agents_gemini_propaganda)) {
  for (j in 1:length(agents_gemini_propaganda)) {
    x <- df_wide_propaganda[[agents_gemini_propaganda[i]]]
    y <- df_wide_propaganda[[agents_gemini_propaganda[j]]]
    mat_gemini_propaganda[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_gemini_propaganda <- as.data.frame(as.table(mat_gemini_propaganda))

loss_gemini_propaganda <- ggplot(hamming_gemini_propaganda, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Gemini agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Copilot -- propaganda links

agents_copilot_propaganda <- grep("Copilot", names(df_wide_propaganda), value = TRUE)

mat_copilot_propaganda <- matrix(0, nrow = length(agents_copilot_propaganda), 
                                 ncol = length(agents_copilot_propaganda),
                                 dimnames = list(agents_copilot_propaganda, agents_copilot_propaganda))

for (i in 1:length(agents_copilot_propaganda)) {
  for (j in 1:length(agents_copilot_propaganda)) {
    x <- df_wide_propaganda[[agents_copilot_propaganda[i]]]
    y <- df_wide_propaganda[[agents_copilot_propaganda[j]]]
    mat_copilot_propaganda[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_copilot_propaganda <- as.data.frame(as.table(mat_copilot_propaganda))

loss_copilot_propaganda <- ggplot(hamming_copilot_propaganda, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Copilot agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))


# Grok -- propaganda links

agents_grok_propaganda <- grep("Grok", names(df_wide_propaganda), value = TRUE)

mat_grok_propaganda <- matrix(0, nrow = length(agents_grok_propaganda), ncol = length(agents_grok_propaganda),
                       dimnames = list(agents_grok_propaganda, agents_grok_propaganda))

for (i in 1:length(agents_grok_propaganda)) {
  for (j in 1:length(agents_grok_propaganda)) {
    x <- df_wide_propaganda[[agents_grok_propaganda[i]]]
    y <- df_wide_propaganda[[agents_grok_propaganda[j]]]
    mat_grok_propaganda[i, j] <- mean(x != y, na.rm = TRUE)
  }
}

hamming_grok_propaganda <- as.data.frame(as.table(mat_grok_propaganda))

loss_grok_propaganda <- ggplot(hamming_grok_propaganda, aes(Var1, Var2, fill = Freq)) +
  geom_tile(color = "white") +
  geom_text(aes(label = sprintf("%.2f", Freq)), size = 3) +
  scale_fill_viridis_c(limits = c(0, 0.4)) +
  labs(title = "Grok agents", x = "", y = "", fill = "") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1),
        plot.title   = element_text(face = "bold", size = 14, hjust = 0.5))

# Merge -- propaganda links

hamming_propaganda <- ggarrange(loss_gpt_propaganda, loss_gemini_propaganda, 
                          loss_copilot_propaganda, loss_grok_propaganda,
                          ncol = 2,
                          nrow = 2)

hamming_propaganda

```


```{r pressure, echo=FALSE}

# AVERAGE HAMMING LOSS

# Average Hamming loss for supporting disinformation

hamming_loss <- function(x, y) mean(x != y, na.rm = TRUE)

avg_hamming_fake_mean <- df2 %>%
  distinct(model, country, instance, agent) %>%
  group_by(model) %>%
  summarise(mean_hamming = {
    agents <- unique(agent[model == first(model)])
    combs <- combn(agents, 2)
    losses <- c()
    for (i in 1:ncol(combs)) {
      a <- combs[1, i]
      b <- combs[2, i]
      losses[i] <- hamming_loss(df_wide_fake[[a]], df_wide_fake[[b]])
    }
    mean(losses, na.rm = TRUE)
  }, .groups = "drop")

stacked_df_fake_mean <- avg_hamming_fake_mean %>%
  mutate(Consistent = 100 * (1 - mean_hamming),
         Inconsistent = 100 * mean_hamming) %>%
  tidyr::pivot_longer(cols = c("Consistent", "Inconsistent"),
                      names_to = "type", values_to = "value") %>%
  mutate(type = factor(type, levels = c("Inconsistent", "Consistent")))

average_stacked_hamming_fake_mean <- ggplot(stacked_df_fake_mean, aes(x = model, y = value, fill = type)) +
  geom_bar(stat = "identity", position = position_stack(), width = 0.9) +
  labs(y = "Percentage of outputs", x = "Chatbot model", fill = "Output") +
  scale_fill_manual(values = c(
    "Inconsistent" = viridis::viridis(2)[2],
    "Consistent"   = viridis::viridis(2)[1]
  )) +
  theme_minimal(base_size = 12) +
  theme(
    panel.grid.major.x = element_blank(),
    axis.text.x = element_text(angle = 25, hjust = 0.5),
    strip.text = element_text(face = "bold"),
    legend.position = "bottom",
    legend.title = element_text(face = "bold"),
        axis.title.x = element_blank()
  )

# Average Hamming loss for referencing Pravda

avg_hamming_pravda_mean <- df2 %>%
  distinct(model, country, instance, agent) %>%
  group_by(model) %>%
  summarise(mean_hamming = {
    agents <- unique(agent[model == first(model)])
    combs <- combn(agents, 2)
    losses <- c()
    for (i in 1:ncol(combs)) {
      a <- combs[1, i]
      b <- combs[2, i]
      losses[i] <- hamming_loss(df_wide_pravda[[a]], df_wide_pravda[[b]])
    }
    mean(losses, na.rm = TRUE)
  }, .groups = "drop")

stacked_df_pravda_mean <- avg_hamming_pravda_mean %>%
  mutate(Consistent = 100 * (1 - mean_hamming),
         Inconsistent = 100 * mean_hamming) %>%
  tidyr::pivot_longer(cols = c("Consistent", "Inconsistent"),
                      names_to = "type", values_to = "value") %>%
  mutate(type = factor(type, levels = c("Inconsistent", "Consistent")))

average_stacked_hamming_pravda_mean <- ggplot(stacked_df_pravda_mean, aes(x = model, y = value, fill = type)) +
  geom_bar(stat = "identity", position = position_stack(), width = .9) +
  labs(y = "Percentage of outputs", x = "Chatbot model", fill = "Output") +
  scale_fill_manual(values = c(
    "Inconsistent" = viridis::viridis(2)[2],
    "Consistent"   = viridis::viridis(2)[1]
  )) +
  theme_minimal(base_size = 12) +
  theme(
    panel.grid.major.x = element_blank(),
    axis.text.x = element_text(angle = 25, hjust = 0.5),
    strip.text = element_text(face = "bold"),
    legend.position = "none",
    legend.title = element_text(face = "bold"),
    axis.title.x = element_blank(),
    axis.title.y = element_blank(),
    axis.text.y = element_blank()
  )

# Merge 

average_stacked_hamming_fake_mean <- average_stacked_hamming_fake_mean +
  ggtitle("Supporting disinformation") +
  theme(
    plot.title = element_text(hjust = 0.5, face = "bold") 
  )

average_stacked_hamming_pravda_mean <- average_stacked_hamming_pravda_mean +
  ggtitle("Pravda links") +
  theme(
    plot.title = element_text(hjust = 0.5, face = "bold") 
  )

average_combined_plot <- average_stacked_hamming_fake_mean +
  average_stacked_hamming_pravda_mean +
  plot_layout(guides = "collect") &
  theme(legend.position = "bottom")

average_combined_plot

# Average Hamming loss for Russian propaganda

avg_hamming_propaganda_mean <- df2 %>%
  distinct(model, country, instance, agent) %>%
  group_by(model) %>%
  summarise(mean_hamming = {
    agents <- unique(agent[model == first(model)])
    combs <- combn(agents, 2)
    losses <- c()
    for (i in 1:ncol(combs)) {
      a <- combs[1, i]
      b <- combs[2, i]
      losses[i] <- hamming_loss(df_wide_propaganda[[a]], df_wide_propaganda[[b]])
    }
    mean(losses, na.rm = TRUE)
  }, .groups = "drop")

stacked_df_propaganda_mean <- avg_hamming_propaganda_mean %>%
  mutate(Consistent = 100 * (1 - mean_hamming),
         Inconsistent = 100 * mean_hamming) %>%
  tidyr::pivot_longer(cols = c("Consistent", "Inconsistent"),
                      names_to = "type", values_to = "value") %>%
  mutate(type = factor(type, levels = c("Inconsistent", "Consistent")))

average_stacked_hamming_propaganda_mean <- ggplot(stacked_df_propaganda_mean, aes(x = model, y = value, fill = type)) +
  geom_bar(stat = "identity", position = position_stack(), width = 0.6) +
  labs(y = "Percentage of outputs", x = "Chatbot model", fill = "Output") +
  scale_fill_manual(values = c(
    "Inconsistent" = viridis::viridis(2)[2],
    "Consistent"   = viridis::viridis(2)[1]
  )) +
  theme_minimal(base_size = 12) +
  theme(
    panel.grid.major.x = element_blank(),
    axis.text.x = element_text(angle = 20, hjust = 1),
    strip.text = element_text(face = "bold"),
    legend.position = "bottom",
    legend.title = element_text(face = "bold"),
        axis.title.x = element_blank()
  )


```


```{r pressure, echo=FALSE}

# Regressions

model_false <- glm(fake ~ model + country, data = df, family = binomial("logit"), method = "brglmFit")

model_pravda <- glm(pravda ~ model + country, data = df, family = binomial("logit"), method = "brglmFit")

model_prompt <- glm(pravda ~ model + country + prompt_type, data = df, family = binomial("logit"), method = "brglmFit")

model_propaganda <- glm(propaganda ~ model + country, data = df, family = binomial("logit"), method = "brglmFit")

stargazer(model_false, model_pravda, model_prompt, model_propaganda, type = "text", style = "apsr")


```

```{r pressure, echo=FALSE}

# Comparison across models

plot_df_brand <- df_2 %>%
  mutate(
    fake   = tolower(trimws(fake)),
    brand  = factor(brand, levels = c("OpenAI", "xAI")),
    model  = factor(model, levels = c("ChatGPT-4o", "ChatGPT-4o-mini", "Grok-2", "Grok-3")),
    fake   = recode(fake,
                    "yes" = "Supporting disinformation",
                    "no"  = "Not supporting disinformation",
                    .default = NA_character_),
    fake   = factor(fake, levels = c("Supporting disinformation", "Not supporting disinformation"))
  ) %>%
  filter(!is.na(fake), !is.na(model), !is.na(brand)) %>%
  count(brand, model, fake, name = "n") %>%
  group_by(brand, model) %>%
  mutate(percentage = 100 * n / sum(n)) %>%
  ungroup()

graph_disinfo_by_brand <- ggplot(plot_df_brand, aes(x = model, y = percentage, fill = fake)) +
  geom_col(width = 0.8, color = "white", linewidth = 0.2) +
  facet_grid(. ~ brand, scales = "free_x", space = "free_x") +
  scale_fill_manual(
    values = c(
      "Supporting disinformation" = "red",
      "Not supporting disinformation" = "blue"
    ),
    name = "Output"
  ) +
  labs(
    x = NULL, y = "Percentage of outputs",
    title = NULL
  ) +
  theme_minimal(base_size = 12) +
  theme(
    panel.grid.major.x = element_blank(),
    axis.text.x = element_text(angle = 20, hjust = 1),
    strip.text = element_text(face = "bold"),
    legend.position = "bottom",
    legend.title = element_text(face = "bold")
  )


```